{
 "cells": [
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": [
    "# 文档加载器的使用 XxxLoader\n",
    "\n",
    "txt文档：TextLoader\n",
    "\n",
    "json文档：JsonLoader\n",
    "\n",
    "csv文档：CsvLoader\n",
    "\n",
    "pdf文档：pyPdfLoader\n",
    "\n",
    "html文档：UnStructuredHtmlLoader\n",
    "\n",
    "...\n",
    "\n",
    "此XxxLoader创建好以后，直接调用其load()方法，得到内存中的一个Document的实例\n",
    "\n",
    "## 1、TextLoader\n",
    "\n"
   ],
   "id": "7ec5e538c5db8d82"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-08-05T09:17:10.232819Z",
     "start_time": "2025-08-05T09:17:10.051099Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 1.导入相关依赖\n",
    "from langchain.document_loaders import TextLoader\n",
    "\n",
    "# 2.定义TextLoader对象，file_path=\".txt的位置\"\n",
    "text_loader = TextLoader(file_path=\"asset/load/01-langchain-utf-8.txt\", encoding=\"utf-8\")\n",
    "\n",
    "# 3.加载\n",
    "docs = text_loader.load()  #返回List列表(Document对象)\n",
    "\n",
    "# 4.打印\n",
    "print(docs)\n",
    "\n",
    "#得到的Document对象，有两个属性。\n",
    "# page_content：存储本身文档的信息\n",
    "# metadata：存储文档的元数据"
   ],
   "id": "aed6547492403d6a",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Document(metadata={'source': 'asset/load/01-langchain-utf-8.txt'}, page_content='LangChain 是一个用于构建基于大语言模型（LLM）应用的开发框架，旨在帮助开发者更高效地集成、管理和增强大语言模型的能力，构建端到端的应用程序。它提供了一套模块化工具和接口，支持从简单的文本生成到复杂的多步骤推理任务')]\n"
     ]
    }
   ],
   "execution_count": 1
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-08-05T09:18:39.933394Z",
     "start_time": "2025-08-05T09:18:39.922647Z"
    }
   },
   "cell_type": "code",
   "source": "print(docs[0].metadata)",
   "id": "3b5c5c8adf6b3e82",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'source': 'asset/load/01-langchain-utf-8.txt'}\n"
     ]
    }
   ],
   "execution_count": 2
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-08-05T09:18:49.079203Z",
     "start_time": "2025-08-05T09:18:49.073198Z"
    }
   },
   "cell_type": "code",
   "source": "print(docs[0].page_content)",
   "id": "d6922e82dada61ae",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LangChain 是一个用于构建基于大语言模型（LLM）应用的开发框架，旨在帮助开发者更高效地集成、管理和增强大语言模型的能力，构建端到端的应用程序。它提供了一套模块化工具和接口，支持从简单的文本生成到复杂的多步骤推理任务\n"
     ]
    }
   ],
   "execution_count": 3
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-08-05T09:19:32.861857Z",
     "start_time": "2025-08-05T09:19:32.842811Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 1.导入相关依赖\n",
    "from langchain.document_loaders import TextLoader\n",
    "\n",
    "# 2.定义TextLoader对象，file_path=\".txt的位置\"\n",
    "text_loader = TextLoader(file_path=\"asset/load/01-langchain-gbk.txt\", encoding=\"gbk\")\n",
    "\n",
    "# 3.加载\n",
    "docs = text_loader.load()  #返回List列表(Document对象)\n",
    "\n",
    "# 4.打印\n",
    "print(docs)\n"
   ],
   "id": "ff3a0fa92be36df5",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Document(metadata={'source': 'asset/load/01-langchain-gbk.txt'}, page_content='LangChain 是一个用于构建基于大语言模型（LLM）应用的开发框架，旨在帮助开发者更高效地集成、管理和增强大语言模型的能力，构建端到端的应用程序。它提供了一套模块化工具和接口，支持从简单的文本生成到复杂的多步骤推理任务')]\n"
     ]
    }
   ],
   "execution_count": 5
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "## 2、PyPDFLoader",
   "id": "df88a90e4fc61924"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-08-05T09:20:49.400942Z",
     "start_time": "2025-08-05T09:20:49.376069Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 1.导入相关的依赖  PyPDFLoader()\n",
    "from langchain.document_loaders import PyPDFLoader\n",
    "\n",
    "# 2.定义PyPDFLoader\n",
    "pdfLoader = PyPDFLoader(file_path=\"asset/load/02-load.pdf\")\n",
    "\n",
    "# 3.加载\n",
    "docs = pdfLoader.load()\n",
    "\n",
    "# print(docs)\n",
    "print(type(docs[0]))\n",
    "\n",
    "# # 4.遍历集合\n",
    "# for doc in docs:\n",
    "#     print(f\"加载的文档:{doc.page_content}\")\n",
    "#"
   ],
   "id": "d4b14b9fd49e823d",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'langchain_core.documents.base.Document'>\n"
     ]
    }
   ],
   "execution_count": 7
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": [
    "## 3、CSVLoader\n",
    "\n"
   ],
   "id": "740e7a57242cbe44"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-08-05T09:23:53.964794Z",
     "start_time": "2025-08-05T09:23:53.953640Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from langchain_community.document_loaders.csv_loader import CSVLoader\n",
    "\n",
    "loader = CSVLoader(file_path=\"asset/load/04-load.csv\")\n",
    "data = loader.load()\n",
    "print(data)\n",
    "\n",
    "print(type(data))  # <class 'list'>\n",
    "print(type(data[0]))  # <class 'langchain_core.documents.base.Document'>\n",
    "print(len(data))  # 4\n",
    "print(data[0].page_content)  # id: 1  title: Introduction to Python ..."
   ],
   "id": "e3f039abd47dab9c",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Document(metadata={'source': 'asset/load/04-load.csv', 'row': 0}, page_content='id: 1\\ntitle: Introduction to Python\\ncontent: Python is a popular programming language.\\nauthor: John Doe'), Document(metadata={'source': 'asset/load/04-load.csv', 'row': 1}, page_content='id: 2\\ntitle: Data Science Basics\\ncontent: Data science involves statistics and machine learning.\\nauthor: Jane Smith'), Document(metadata={'source': 'asset/load/04-load.csv', 'row': 2}, page_content='id: 3\\ntitle: Web Development\\ncontent: HTML, CSS and JavaScript are core web technologies.\\nauthor: Mike Johnson'), Document(metadata={'source': 'asset/load/04-load.csv', 'row': 3}, page_content='id: 4\\ntitle: Artificial Intelligence\\ncontent: AI is transforming many industries.\\nauthor: Sarah Williams')]\n",
      "<class 'list'>\n",
      "<class 'langchain_core.documents.base.Document'>\n",
      "4\n",
      "id: 1\n",
      "title: Introduction to Python\n",
      "content: Python is a popular programming language.\n",
      "author: John Doe\n"
     ]
    }
   ],
   "execution_count": 8
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-08-05T09:24:51.331985Z",
     "start_time": "2025-08-05T09:24:51.312625Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from langchain_community.document_loaders.csv_loader import CSVLoader\n",
    "\n",
    "loader = CSVLoader(\n",
    "    file_path=\"./asset/load/04-load.csv\",\n",
    "    source_column='author'  #影响的是metadata中的source字段的值\n",
    ")\n",
    "\n",
    "data = loader.load()\n",
    "print(data)"
   ],
   "id": "eb3e2f180af81346",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Document(metadata={'source': 'John Doe', 'row': 0}, page_content='id: 1\\ntitle: Introduction to Python\\ncontent: Python is a popular programming language.\\nauthor: John Doe'), Document(metadata={'source': 'Jane Smith', 'row': 1}, page_content='id: 2\\ntitle: Data Science Basics\\ncontent: Data science involves statistics and machine learning.\\nauthor: Jane Smith'), Document(metadata={'source': 'Mike Johnson', 'row': 2}, page_content='id: 3\\ntitle: Web Development\\ncontent: HTML, CSS and JavaScript are core web technologies.\\nauthor: Mike Johnson'), Document(metadata={'source': 'Sarah Williams', 'row': 3}, page_content='id: 4\\ntitle: Artificial Intelligence\\ncontent: AI is transforming many industries.\\nauthor: Sarah Williams')]\n"
     ]
    }
   ],
   "execution_count": 9
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "## 4、JSONLoader",
   "id": "f2f8dd0bf492d632"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-08-05T09:30:16.844343Z",
     "start_time": "2025-08-05T09:30:16.827777Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 1.导入依赖\n",
    "from  langchain_community.document_loaders import JSONLoader\n",
    "from pprint import pprint\n",
    "\n",
    "# 2.定义JSONLoader对象\n",
    "# 错误的\n",
    "# json_loader=JSONLoader(file_path=\"asset/load/03-load.json\")\n",
    "\n",
    "# 情况1\n",
    "# json_loader=JSONLoader(\n",
    "#     file_path=\"asset/load/03-load.json\",\n",
    "#     jq_schema=\".\", #直接提取完整的JSON对象（包括所有字段）\n",
    "#     text_content=False #保持原始 JSON 结构，将提取的数据转换为SON字符串存入page_content字段中\n",
    "# )\n",
    "\n",
    "# 情况2\n",
    "# .messages[].content:遍历.messages[]中所有元素 从每一个元素中提取.content字段\n",
    "json_loader=JSONLoader(\n",
    "    file_path=\"asset/load/03-load.json\",\n",
    "    jq_schema=\".messages[].content\")\n",
    "\n",
    "# 3.加载\n",
    "docs = json_loader.load()\n",
    "pprint(docs)\n",
    "\n",
    "# 4.提取content中指定字符数的内容\n",
    "# print(docs[0].page_content[:10])"
   ],
   "id": "846eacd1d4a1dd35",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Document(metadata={'source': 'D:\\\\code\\\\workspace_pycharm_llm\\\\langchain_tutorial\\\\chapter07_RAG\\\\asset\\\\load\\\\03-load.json', 'seq_num': 1}, page_content='Hello, how are you today?'),\n",
      " Document(metadata={'source': 'D:\\\\code\\\\workspace_pycharm_llm\\\\langchain_tutorial\\\\chapter07_RAG\\\\asset\\\\load\\\\03-load.json', 'seq_num': 2}, page_content=\"I'm doing well, thanks for asking!\"),\n",
      " Document(metadata={'source': 'D:\\\\code\\\\workspace_pycharm_llm\\\\langchain_tutorial\\\\chapter07_RAG\\\\asset\\\\load\\\\03-load.json', 'seq_num': 3}, page_content='Would you like to meet for lunch?'),\n",
      " Document(metadata={'source': 'D:\\\\code\\\\workspace_pycharm_llm\\\\langchain_tutorial\\\\chapter07_RAG\\\\asset\\\\load\\\\03-load.json', 'seq_num': 4}, page_content='Sure, that sounds great!')]\n"
     ]
    }
   ],
   "execution_count": 11
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-08-05T09:33:21.420932Z",
     "start_time": "2025-08-05T09:33:21.396483Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 1.导入相关依赖\n",
    "from langchain_community.document_loaders import JSONLoader\n",
    "from pprint import pprint\n",
    "\n",
    "# 2.定义json文件的路径\n",
    "file_path = 'asset/load/03-response.json'\n",
    "\n",
    "# 3.定义JSONLoader对象\n",
    "# 提取嵌套在 data.items[].content 的文本，并保留其他字段作为元数据\n",
    "loader = JSONLoader(\n",
    "    file_path=file_path,\n",
    "    jq_schema=\".data.items[]\",  # 先定位到数组条目\n",
    "    content_key=\".content\",  # 再从条目中提取 content 字段\n",
    "    is_content_key_jq_parsable=True  # 用jq解析content_key\n",
    ")\n",
    "\n",
    "# loader = JSONLoader(\n",
    "#     file_path=file_path,\n",
    "#     jq_schema=\".data.items[].content\",  # 先定位到数组条目\n",
    "#     # content_key=\".content\",  # 再从条目中提取 content 字段\n",
    "#     # is_content_key_jq_parsable=True  # 用jq解析content_key\n",
    "# )\n",
    "\n",
    "# 4.加载\n",
    "data = loader.load()\n",
    "pprint(data)\n",
    "\n",
    "# pprint(data[0].page_content)"
   ],
   "id": "173f478cccca2b4",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Document(metadata={'source': 'D:\\\\code\\\\workspace_pycharm_llm\\\\langchain_tutorial\\\\chapter07_RAG\\\\asset\\\\load\\\\03-response.json', 'seq_num': 1}, page_content='This article explains how to parse API responses...'),\n",
      " Document(metadata={'source': 'D:\\\\code\\\\workspace_pycharm_llm\\\\langchain_tutorial\\\\chapter07_RAG\\\\asset\\\\load\\\\03-response.json', 'seq_num': 2}, page_content='Learn to handle nested structures with...'),\n",
      " Document(metadata={'source': 'D:\\\\code\\\\workspace_pycharm_llm\\\\langchain_tutorial\\\\chapter07_RAG\\\\asset\\\\load\\\\03-response.json', 'seq_num': 3}, page_content='Best practices for preserving metadata...')]\n"
     ]
    }
   ],
   "execution_count": 13
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
